{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp mtl_model.mmoe\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MMoE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "from typing import Dict, Tuple\n",
    "\n",
    "import tensorflow as tf\n",
    "from m3tl.base_params import BaseParams\n",
    "from m3tl.mtl_model.base import MTLBase\n",
    "from m3tl.utils import get_phase\n",
    "\n",
    "\n",
    "class MMoE(MTLBase):\n",
    "    def __init__(self, params: BaseParams, name:str):\n",
    "        super(MMoE, self).__init__(params, name)\n",
    "        self.num_experts = self.params.get('num_experts', 8)\n",
    "        self.num_experts_units = self.params.get('num_experts_units', 128)\n",
    "        self.problem_list = self.params.problem_list\n",
    "        self.gate_dict = {\n",
    "            problem: tf.keras.layers.Dense(self.num_experts, activation='softmax') for problem in self.problem_list\n",
    "        }\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        features_input_shape, hidden_feature_input_shape = input_shape\n",
    "        pooled_shape = hidden_feature_input_shape['all']['pooled']\n",
    "        self.experts_kernel = self.add_weight(\n",
    "            name='experts_kernel',\n",
    "            shape=(pooled_shape[-1], self.num_experts_units, self.num_experts)\n",
    "        )\n",
    "        # add leading dims to support braodcasting\n",
    "        self.experts_bias = self.add_weight(\n",
    "            name='experts_bias',\n",
    "            shape=(1, 1, self.num_experts_units, self.num_experts)\n",
    "        )\n",
    "\n",
    "    def call(self, inputs: Tuple[Dict[str, tf.Tensor]]):\n",
    "        mode = get_phase()\n",
    "        features, hidden_features = inputs\n",
    "        all_features, all_hidden_features = self.extract_feature('all', feature_dict=features, hidden_feature_dict=hidden_features)\n",
    "\n",
    "        # get seq outputs\n",
    "        # [batch_size, seq_len, hidden_size]\n",
    "        seq_hidden = all_hidden_features['seq']\n",
    "        # [batch_size, seq_len, num_expert_units, num_experts]\n",
    "        experts_outputs = tf.tensordot(seq_hidden, self.experts_kernel, axes=[2, 0]) + self.experts_bias\n",
    "\n",
    "        experts_output_dict = {\n",
    "            'pooled': experts_outputs[:, 0, :, :],\n",
    "            'seq': experts_outputs\n",
    "        }\n",
    "\n",
    "        # per problem gating\n",
    "        # we can save a little bit of computation by extract per problem features first\n",
    "        features_per_problem, hidden_features_per_problem = {}, {}\n",
    "        for problem, gate_net in self.gate_dict.items():\n",
    "            features_per_problem[problem], problem_experts_output = self.extract_feature(\n",
    "                extract_problem=problem, feature_dict=all_features, hidden_feature_dict=experts_output_dict\n",
    "            )\n",
    "            _, problem_hidden_features = self.extract_feature(\n",
    "                extract_problem=problem, feature_dict=all_features, hidden_feature_dict=all_hidden_features\n",
    "            )\n",
    "\n",
    "            # apply gating\n",
    "            # [problem_batch_size, seq_len, 1, num_experts]\n",
    "            experts_weight = gate_net(problem_hidden_features['seq'])\n",
    "            experts_weight = tf.expand_dims(experts_weight, axis=2)\n",
    "            # [problem_batch_size, seq_len, num_expert_units, num_experts]\n",
    "            expert_output_per_problem = problem_experts_output['seq']\n",
    "\n",
    "            # [problem_batch_size, seq_len, num_expert_units]\n",
    "            gated_experts_output = tf.reduce_mean(experts_weight*expert_output_per_problem, axis=-1)\n",
    "            hidden_features_per_problem[problem] = {\n",
    "                'pooled': gated_experts_output[:, 0, :],\n",
    "                'seq': gated_experts_output\n",
    "            }\n",
    "\n",
    "        return features_per_problem, hidden_features_per_problem\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-06-06 00:08:22.007 | INFO     | m3tl.base_params:register_multiple_problems:476 - Adding new problem weibo_fake_ner, problem type: seq_tag\n",
      "2021-06-06 00:08:22.008 | INFO     | m3tl.base_params:register_multiple_problems:476 - Adding new problem weibo_fake_multi_cls, problem type: multi_cls\n",
      "2021-06-06 00:08:22.009 | INFO     | m3tl.base_params:register_multiple_problems:476 - Adding new problem weibo_fake_cls, problem type: cls\n",
      "2021-06-06 00:08:22.009 | INFO     | m3tl.base_params:register_multiple_problems:476 - Adding new problem weibo_masklm, problem type: masklm\n",
      "2021-06-06 00:08:22.009 | INFO     | m3tl.base_params:register_multiple_problems:476 - Adding new problem weibo_fake_regression, problem type: regression\n",
      "2021-06-06 00:08:22.010 | INFO     | m3tl.base_params:register_multiple_problems:476 - Adding new problem weibo_fake_vector_fit, problem type: vector_fit\n",
      "2021-06-06 00:08:22.011 | INFO     | m3tl.base_params:register_multiple_problems:476 - Adding new problem weibo_premask_mlm, problem type: premask_mlm\n",
      "2021-06-06 00:08:22.012 | WARNING  | m3tl.base_params:prepare_dir:314 - bert_config not exists. will load model from huggingface checkpoint.\n"
     ]
    }
   ],
   "source": [
    "# hide\n",
    "from m3tl.test_base import TestBase\n",
    "from m3tl.special_tokens import TRAIN, EVAL, PREDICT\n",
    "\n",
    "tb = TestBase()\n",
    "mmoe = MMoE(params=tb.params, name='test_mmoe')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hide\n",
    "# features, hidden_features = tb.get_one_batch_body_model_output()\n",
    "# mmoe = MMoE(params=tb.params, name='test_mmoe')\n",
    "# features_per_problem, hidden_features_per_problem = mmoe((features, hidden_features), TRAIN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-06-06 00:08:28.196 | WARNING  | m3tl.read_write_tfrecord:chain_processed_data:237 - Chaining problems with & may consume a lot of memory if data is not pyspark RDD.\n",
      "2021-06-06 00:08:28.222 | DEBUG    | m3tl.read_write_tfrecord:_write_fn:134 - Writing /tmp/tmpqvmqxfcx/weibo_fake_cls_weibo_fake_ner_weibo_fake_regression_weibo_fake_vector_fit/train_00000.tfrecord\n",
      "2021-06-06 00:08:28.358 | WARNING  | m3tl.read_write_tfrecord:chain_processed_data:237 - Chaining problems with & may consume a lot of memory if data is not pyspark RDD.\n",
      "2021-06-06 00:08:28.384 | DEBUG    | m3tl.read_write_tfrecord:_write_fn:134 - Writing /tmp/tmpqvmqxfcx/weibo_fake_cls_weibo_fake_ner_weibo_fake_regression_weibo_fake_vector_fit/eval_00000.tfrecord\n",
      "2021-06-06 00:08:28.439 | DEBUG    | m3tl.read_write_tfrecord:_write_fn:134 - Writing /tmp/tmpqvmqxfcx/weibo_fake_multi_cls/train_00000.tfrecord\n",
      "2021-06-06 00:08:28.486 | DEBUG    | m3tl.read_write_tfrecord:_write_fn:134 - Writing /tmp/tmpqvmqxfcx/weibo_fake_multi_cls/eval_00000.tfrecord\n",
      "2021-06-06 00:08:28.578 | DEBUG    | m3tl.read_write_tfrecord:_write_fn:134 - Writing /tmp/tmpqvmqxfcx/weibo_masklm/train_00000.tfrecord\n",
      "2021-06-06 00:08:28.627 | DEBUG    | m3tl.read_write_tfrecord:_write_fn:134 - Writing /tmp/tmpqvmqxfcx/weibo_masklm/eval_00000.tfrecord\n",
      "2021-06-06 00:08:28.692 | DEBUG    | m3tl.read_write_tfrecord:_write_fn:134 - Writing /tmp/tmpqvmqxfcx/weibo_premask_mlm/train_00000.tfrecord\n",
      "2021-06-06 00:08:28.756 | DEBUG    | m3tl.read_write_tfrecord:_write_fn:134 - Writing /tmp/tmpqvmqxfcx/weibo_premask_mlm/eval_00000.tfrecord\n",
      "2021-06-06 00:08:29.718 | INFO     | m3tl.input_fn:train_eval_input_fn:56 - sampling weights: \n",
      "2021-06-06 00:08:29.719 | INFO     | m3tl.input_fn:train_eval_input_fn:57 - {\n",
      "    \"weibo_fake_cls_weibo_fake_ner_weibo_fake_regression_weibo_fake_vector_fit\": 0.2564102564102564,\n",
      "    \"weibo_fake_multi_cls\": 0.2564102564102564,\n",
      "    \"weibo_masklm\": 0.23076923076923078,\n",
      "    \"weibo_premask_mlm\": 0.2564102564102564\n",
      "}\n",
      "404 Client Error: Not Found for url: https://huggingface.co/voidful/albert_chinese_tiny/resolve/main/tf_model.h5\n",
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFAlbertModel: ['predictions.dense.bias', 'predictions.decoder.bias', 'predictions.LayerNorm.weight', 'predictions.bias', 'predictions.decoder.weight', 'predictions.dense.weight', 'predictions.LayerNorm.bias']\n",
      "- This IS expected if you are initializing TFAlbertModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFAlbertModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFAlbertModel were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFAlbertModel for predictions without further training.\n",
      "2021-06-06 00:08:34.538 | CRITICAL | m3tl.modeling:__init__:57 - Modal Type id mapping: \n",
      " {\n",
      "    \"array\": 0,\n",
      "    \"cate\": 1,\n",
      "    \"text\": 2\n",
      "}\n",
      "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
      "WARNING: AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7f5dc1bdc5e0>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: module, class, method, function, traceback, frame, or code object was expected, got cython_function_or_method\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
      "2021-06-06 00:08:40.061 | WARNING  | m3tl.problem_types.masklm:__init__:40 - Share embedding is enabled but hidden_size != embedding_size\n",
      "2021-06-06 00:08:40.869 | INFO     | m3tl.input_fn:train_eval_input_fn:56 - sampling weights: \n",
      "2021-06-06 00:08:40.870 | INFO     | m3tl.input_fn:train_eval_input_fn:57 - {\n",
      "    \"weibo_fake_cls_weibo_fake_ner_weibo_fake_regression_weibo_fake_vector_fit\": 0.2564102564102564,\n",
      "    \"weibo_fake_multi_cls\": 0.2564102564102564,\n",
      "    \"weibo_masklm\": 0.23076923076923078,\n",
      "    \"weibo_premask_mlm\": 0.2564102564102564\n",
      "}\n",
      "2021-06-06 00:08:42.344 | INFO     | m3tl.input_fn:train_eval_input_fn:56 - sampling weights: \n",
      "2021-06-06 00:08:42.345 | INFO     | m3tl.input_fn:train_eval_input_fn:57 - {\n",
      "    \"weibo_fake_cls_weibo_fake_ner_weibo_fake_regression_weibo_fake_vector_fit\": 0.2564102564102564,\n",
      "    \"weibo_fake_multi_cls\": 0.2564102564102564,\n",
      "    \"weibo_masklm\": 0.23076923076923078,\n",
      "    \"weibo_premask_mlm\": 0.2564102564102564\n",
      "}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "938, 0.19856867, 0.20290618, 0.1969807 ],\n",
       "         [0.19851914, 0.20053253, 0.20000899, 0.20155519, 0.19938414],\n",
       "         [0.19841829, 0.20379424, 0.19912775, 0.20094569, 0.19771394],\n",
       "         [0.19755088, 0.20352331, 0.19960462, 0.20161344, 0.19770773],\n",
       "         [0.19965973, 0.20355953, 0.19870031, 0.20076627, 0.19731414],\n",
       "         [0.19917299, 0.20364495, 0.19787578, 0.20028768, 0.19901864],\n",
       "         [0.19916093, 0.2036096 , 0.19698974, 0.20146157, 0.19877811],\n",
       "         [0.20121635, 0.2021869 , 0.1968712 , 0.20115578, 0.19856976],\n",
       "         [0.19932503, 0.20335594, 0.19703469, 0.20196989, 0.19831455],\n",
       "         [0.20464312, 0.19796327, 0.1953807 , 0.20584385, 0.19616906],\n",
       "         [0.20757926, 0.1974735 , 0.19958356, 0.2029948 , 0.19236887],\n",
       "         [0.20556863, 0.19863138, 0.19653077, 0.20520416, 0.19406508],\n",
       "         [0.20250417, 0.2012576 , 0.19577971, 0.20254788, 0.19791062],\n",
       "         [0.20522092, 0.19764833, 0.19517438, 0.20537657, 0.19657977]],\n",
       " \n",
       "        [[0.19836774, 0.20244278, 0.20008492, 0.2019677 , 0.19713686],\n",
       "         [0.19961669, 0.20150408, 0.1998486 , 0.20214032, 0.19689025],\n",
       "         [0.2022711 , 0.20363222, 0.19776784, 0.20072876, 0.19560014],\n",
       "         [0.19775748, 0.20422468, 0.20282844, 0.20059206, 0.1945973 ],\n",
       "         [0.1991173 , 0.20404792, 0.19870779, 0.20200224, 0.19612476],\n",
       "         [0.19929723, 0.20304331, 0.20090692, 0.20141026, 0.1953422 ],\n",
       "         [0.19883354, 0.20409949, 0.19843623, 0.20015346, 0.19847734],\n",
       "         [0.19988877, 0.20391858, 0.19846302, 0.19920254, 0.19852716],\n",
       "         [0.19993025, 0.20294055, 0.19783531, 0.20022842, 0.19906549],\n",
       "         [0.19833767, 0.20344633, 0.19901831, 0.20083638, 0.19836129],\n",
       "         [0.19919834, 0.20361729, 0.19898619, 0.20033315, 0.19786508],\n",
       "         [0.19976747, 0.20267943, 0.19630682, 0.20250043, 0.19874589],\n",
       "         [0.1991765 , 0.20361958, 0.19836096, 0.2012691 , 0.1975739 ],\n",
       "         [0.20123802, 0.203904  , 0.19526707, 0.2021192 , 0.19747175],\n",
       "         [0.20015687, 0.20240454, 0.19667554, 0.20286824, 0.19789483],\n",
       "         [0.20263492, 0.19696169, 0.1992328 , 0.20569447, 0.19547607],\n",
       "         [0.20880836, 0.19920379, 0.19584113, 0.20321654, 0.19293018],\n",
       "         [0.20673019, 0.19531079, 0.19635323, 0.20500833, 0.1965975 ],\n",
       "         [0.20193566, 0.20076922, 0.19601363, 0.202994  , 0.19828741],\n",
       "         [0.20350094, 0.19756265, 0.19682243, 0.20383115, 0.19828282]],\n",
       " \n",
       "        [[0.1989033 , 0.20242202, 0.19941111, 0.20068789, 0.19857572],\n",
       "         [0.19894892, 0.20266297, 0.19990753, 0.20023824, 0.19824226],\n",
       "         [0.20160285, 0.20402905, 0.19677445, 0.2019129 , 0.19568078],\n",
       "         [0.20007981, 0.20220384, 0.20246275, 0.19721206, 0.19804157],\n",
       "         [0.20079796, 0.20280708, 0.19794784, 0.20157589, 0.19687115],\n",
       "         [0.1979778 , 0.20186947, 0.20114255, 0.20239058, 0.19661963],\n",
       "         [0.19969599, 0.20327374, 0.19663824, 0.20159101, 0.19880107],\n",
       "         [0.1996346 , 0.20218803, 0.19728576, 0.20158646, 0.19930525],\n",
       "         [0.1989107 , 0.20269686, 0.1979153 , 0.20074512, 0.19973199],\n",
       "         [0.19792819, 0.20398407, 0.19921209, 0.20136459, 0.19751108],\n",
       "         [0.19860297, 0.20276321, 0.1996849 , 0.20113122, 0.19781762],\n",
       "         [0.19926481, 0.20283261, 0.1980475 , 0.20165074, 0.19820437],\n",
       "         [0.19994445, 0.2035729 , 0.1970342 , 0.20171095, 0.19773747],\n",
       "         [0.20026395, 0.20402052, 0.19853272, 0.20092627, 0.19625653],\n",
       "         [0.19959094, 0.20252916, 0.19864167, 0.20189792, 0.19734031],\n",
       "         [0.20446225, 0.19585486, 0.19976954, 0.2017707 , 0.19814272],\n",
       "         [0.2042397 , 0.19919345, 0.19900247, 0.20177358, 0.1957908 ],\n",
       "         [0.20457833, 0.19679105, 0.1977188 , 0.20347403, 0.19743785],\n",
       "         [0.20340645, 0.20145924, 0.19526175, 0.2031556 , 0.19671696],\n",
       "         [0.20629321, 0.19796479, 0.19585443, 0.20410357, 0.19578399]],\n",
       " \n",
       "        [[0.19906035, 0.20331447, 0.20042004, 0.20110245, 0.19610262],\n",
       "         [0.20023331, 0.20358206, 0.1980071 , 0.20101014, 0.19716738],\n",
       "         [0.20253591, 0.20103626, 0.1991544 , 0.2016408 , 0.19563267],\n",
       "         [0.19723904, 0.20671462, 0.20099089, 0.19837256, 0.19668297],\n",
       "         [0.20011401, 0.20235752, 0.19923864, 0.20185046, 0.1964393 ],\n",
       "         [0.1998523 , 0.20277953, 0.19996546, 0.20215952, 0.19524318],\n",
       "         [0.19972946, 0.2021208 , 0.19839887, 0.20200926, 0.19774158],\n",
       "         [0.19962935, 0.2029988 , 0.19834837, 0.20098472, 0.19803867],\n",
       "         [0.19946815, 0.2017678 , 0.19856694, 0.20124865, 0.1989484 ],\n",
       "         [0.19847351, 0.20422333, 0.20005007, 0.19902726, 0.19822587],\n",
       "         [0.2002282 , 0.20270123, 0.19810522, 0.20088899, 0.19807635],\n",
       "         [0.19845977, 0.20514432, 0.19752909, 0.2012042 , 0.19766264],\n",
       "         [0.19936176, 0.2023697 , 0.19839117, 0.20170367, 0.19817375],\n",
       "         [0.19973612, 0.20285854, 0.19790831, 0.20228557, 0.19721143],\n",
       "         [0.20036411, 0.2032229 , 0.19774216, 0.20182964, 0.19684114],\n",
       "         [0.20561492, 0.19577344, 0.19487096, 0.20594732, 0.19779329],\n",
       "         [0.20390104, 0.20023221, 0.19876903, 0.20264602, 0.1944517 ],\n",
       "         [0.20829558, 0.19614887, 0.19641283, 0.20306984, 0.19607292],\n",
       "         [0.20262606, 0.19997777, 0.19600473, 0.20266324, 0.19872816],\n",
       "         [0.20689452, 0.19780518, 0.19497915, 0.20448107, 0.19584003]],\n",
       " \n",
       "        [[0.19877581, 0.20203598, 0.20156904, 0.20178337, 0.19583589],\n",
       "         [0.20047517, 0.20296223, 0.19853055, 0.20177723, 0.19625479],\n",
       "         [0.20187937, 0.20296383, 0.1981283 , 0.20050733, 0.19652112],\n",
       "         [0.1995587 , 0.20385745, 0.20230663, 0.19714683, 0.19713038],\n",
       "         [0.20043226, 0.20096879, 0.19927265, 0.20320868, 0.1961176 ],\n",
       "         [0.19790019, 0.20239091, 0.20097452, 0.20235433, 0.19637999],\n",
       "         [0.1995297 , 0.2037926 , 0.19639638, 0.20190766, 0.19837372],\n",
       "         [0.20109446, 0.20358141, 0.1970614 , 0.20056307, 0.19769967],\n",
       "         [0.19930562, 0.20469242, 0.19819373, 0.20086937, 0.1969389 ],\n",
       "         [0.20126796, 0.20313129, 0.19762968, 0.2000935 , 0.19787762],\n",
       "         [0.19877233, 0.20235962, 0.199829  , 0.20171168, 0.1973274 ],\n",
       "         [0.19974315, 0.2031718 , 0.19853885, 0.19977368, 0.19877246],\n",
       "         [0.19912419, 0.20312755, 0.19709876, 0.20231663, 0.19833282],\n",
       "         [0.19912301, 0.20261103, 0.19875477, 0.20258684, 0.19692434],\n",
       "         [0.20130548, 0.2028848 , 0.19683997, 0.20217478, 0.19679497],\n",
       "         [0.20518695, 0.19642217, 0.19877507, 0.20519497, 0.19442086],\n",
       "         [0.203721  , 0.19862008, 0.20046039, 0.20310205, 0.19409648],\n",
       "         [0.2023742 , 0.19709563, 0.1976898 , 0.20316485, 0.19967555],\n",
       "         [0.20251119, 0.20216016, 0.19701628, 0.20151727, 0.19679514],\n",
       "         [0.20543991, 0.19678551, 0.19717517, 0.20270178, 0.19789754]],\n",
       " \n",
       "        [[0.19857071, 0.20338227, 0.2004304 , 0.20125388, 0.19636273],\n",
       "         [0.20069183, 0.20324968, 0.19829714, 0.20004302, 0.19771829],\n",
       "         [0.20120445, 0.20236318, 0.19941883, 0.2010447 , 0.19596885],\n",
       "         [0.19777618, 0.20619044, 0.20309375, 0.19629955, 0.19664009],\n",
       "         [0.20005369, 0.20305762, 0.19761841, 0.2017268 , 0.19754349],\n",
       "         [0.1987216 , 0.20217782, 0.20078883, 0.20213066, 0.19618104],\n",
       "         [0.19897729, 0.20261565, 0.19857313, 0.20304365, 0.19679031],\n",
       "         [0.2003561 , 0.20508066, 0.1959141 , 0.20030236, 0.19834672],\n",
       "         [0.20036812, 0.20276898, 0.19877341, 0.2008901 , 0.19719933],\n",
       "         [0.20011677, 0.20301007, 0.1993111 , 0.20055473, 0.19700742],\n",
       "         [0.19872859, 0.20378423, 0.1987996 , 0.20108391, 0.19760369],\n",
       "         [0.1988734 , 0.20421812, 0.19799821, 0.20173381, 0.19717647],\n",
       "         [0.1994564 , 0.2035487 , 0.19675227, 0.20140629, 0.19883636],\n",
       "         [0.19912948, 0.20350116, 0.19869691, 0.20078775, 0.19788465],\n",
       "         [0.19929998, 0.20409502, 0.19738342, 0.20168611, 0.1975355 ],\n",
       "         [0.2050246 , 0.19673657, 0.19936867, 0.20522282, 0.19364737],\n",
       "         [0.20347258, 0.20159225, 0.197981  , 0.20180693, 0.19514722],\n",
       "         [0.20636067, 0.19767317, 0.19502732, 0.20473813, 0.19620074],\n",
       "         [0.20309332, 0.20172699, 0.19627818, 0.20130304, 0.19759849],\n",
       "         [0.20434485, 0.1974325 , 0.19641249, 0.20365886, 0.19815129]]],\n",
       "       dtype=float32)>,\n",
       " 'weibo_fake_cls': <tf.Tensor: shape=(10, 2), dtype=float32, numpy=\n",
       " array([[0.51099646, 0.48900354],\n",
       "        [0.5088481 , 0.4911519 ],\n",
       "        [0.5110601 , 0.48893985],\n",
       "        [0.5137314 , 0.4862686 ],\n",
       "        [0.5082876 , 0.4917124 ],\n",
       "        [0.50974226, 0.4902578 ],\n",
       "        [0.5103374 , 0.48966256],\n",
       "        [0.51330703, 0.48669302],\n",
       "        [0.51055753, 0.48944247],\n",
       "        [0.5087347 , 0.49126527]], dtype=float32)>,\n",
       " 'weibo_fake_regression': <tf.Tensor: shape=(10, 1), dtype=float32, numpy=\n",
       " array([[0.00255248],\n",
       "        [0.00376823],\n",
       "        [0.00255109],\n",
       "        [0.00181741],\n",
       "        [0.00279811],\n",
       "        [0.00216199],\n",
       "        [0.00321892],\n",
       "        [0.00343737],\n",
       "        [0.00320243],\n",
       "        [0.00254907]], dtype=float32)>,\n",
       " 'weibo_fake_vector_fit': <tf.Tensor: shape=(10, 10), dtype=float32, numpy=\n",
       " array([[-9.02229268e-03,  7.39284558e-03,  1.08064245e-03,\n",
       "          5.13076026e-04, -2.69261245e-02,  1.52384024e-02,\n",
       "          9.80578625e-06, -1.04339067e-02,  1.15968492e-02,\n",
       "         -1.79917999e-02],\n",
       "        [-7.76215177e-03,  7.68898614e-03,  6.15899800e-04,\n",
       "          4.41597454e-04, -2.61623487e-02,  1.57853514e-02,\n",
       "          4.18336385e-05, -1.16196275e-02,  1.16791660e-02,\n",
       "         -1.74203552e-02],\n",
       "        [-7.93645252e-03,  7.56629836e-03,  4.18359617e-04,\n",
       "          1.07309490e-03, -2.63709296e-02,  1.51925562e-02,\n",
       "          1.19489472e-04, -9.78218485e-03,  1.16076712e-02,\n",
       "         -1.71353314e-02],\n",
       "        [-9.32611618e-03,  6.92040334e-03,  1.40271033e-03,\n",
       "          7.54753069e-04, -2.72467565e-02,  1.51182739e-02,\n",
       "          2.40434441e-04, -1.01626292e-02,  1.15655717e-02,\n",
       "         -1.83535498e-02],\n",
       "        [-9.40732751e-03,  7.14135915e-03,  1.70587818e-03,\n",
       "          6.02096319e-04, -2.70979051e-02,  1.55698527e-02,\n",
       "         -4.77708345e-05, -9.79342125e-03,  1.14476113e-02,\n",
       "         -1.81727130e-02],\n",
       "        [-8.66893772e-03,  7.03166099e-03,  1.11604494e-03,\n",
       "          8.70706514e-04, -2.66649425e-02,  1.52423568e-02,\n",
       "          6.62923849e-05, -9.95723065e-03,  1.14202118e-02,\n",
       "         -1.78365540e-02],\n",
       "        [-7.59753864e-03,  7.18383910e-03,  1.28003373e-03,\n",
       "          8.88329290e-04, -2.67012976e-02,  1.64594688e-02,\n",
       "         -3.76972457e-04, -1.08375251e-02,  1.16014732e-02,\n",
       "         -1.75816361e-02],\n",
       "        [-8.37488286e-03,  7.72143807e-03,  1.82708434e-03,\n",
       "          8.26241798e-04, -2.68343668e-02,  1.64599083e-02,\n",
       "         -2.60863308e-04, -1.02447271e-02,  1.12176165e-02,\n",
       "         -1.78879574e-02],\n",
       "        [-5.97112672e-03,  7.97777995e-03,  5.35757921e-04,\n",
       "          6.54387521e-04, -2.67094504e-02,  1.62626300e-02,\n",
       "          1.55073110e-04, -1.15408283e-02,  1.20602548e-02,\n",
       "         -1.65157709e-02],\n",
       "        [-7.95227848e-03,  7.47575751e-03,  1.08113268e-03,\n",
       "          2.82595516e-04, -2.64341012e-02,  1.58797298e-02,\n",
       "         -3.99145094e-04, -1.06483987e-02,  1.13757895e-02,\n",
       "         -1.74186099e-02]], dtype=float32)>,\n",
       " 'weibo_fake_multi_cls': <tf.Tensor: shape=(6, 4), dtype=float32, numpy=\n",
       " array([[0.5067333 , 0.50619525, 0.49950248, 0.49568552],\n",
       "        [0.5034789 , 0.50199497, 0.49818388, 0.49488005],\n",
       "        [0.5053423 , 0.50307757, 0.5006657 , 0.49778852],\n",
       "        [0.50446665, 0.5024047 , 0.49949086, 0.49759603],\n",
       "        [0.5026937 , 0.5050054 , 0.50080305, 0.49917454],\n",
       "        [0.5050762 , 0.50390977, 0.49999738, 0.49891517]], dtype=float32)>,\n",
       " 'weibo_masklm': <tf.Tensor: shape=(8, 20, 21128), dtype=float32, numpy=\n",
       " array([[[4.7337358e-05, 4.7273454e-05, 4.7312176e-05, ...,\n",
       "          4.7328740e-05, 4.7331196e-05, 4.7377514e-05],\n",
       "         [4.7333189e-05, 4.7277579e-05, 4.7294991e-05, ...,\n",
       "          4.7329293e-05, 4.7347294e-05, 4.7365927e-05],\n",
       "         [4.7348254e-05, 4.7312511e-05, 4.7339032e-05, ...,\n",
       "          4.7303827e-05, 4.7334979e-05, 4.7353446e-05],\n",
       "         ...,\n",
       "         [4.7348254e-05, 4.7312511e-05, 4.7339032e-05, ...,\n",
       "          4.7303827e-05, 4.7334979e-05, 4.7353446e-05],\n",
       "         [4.7348254e-05, 4.7312511e-05, 4.7339032e-05, ...,\n",
       "          4.7303827e-05, 4.7334979e-05, 4.7353446e-05],\n",
       "         [4.7348254e-05, 4.7312511e-05, 4.7339032e-05, ...,\n",
       "          4.7303827e-05, 4.7334979e-05, 4.7353446e-05]],\n",
       " \n",
       "        [[4.7324105e-05, 4.7276921e-05, 4.7299392e-05, ...,\n",
       "          4.7295442e-05, 4.7337540e-05, 4.7375543e-05],\n",
       "         [4.7341349e-05, 4.7285394e-05, 4.7300771e-05, ...,\n",
       "          4.7327470e-05, 4.7341655e-05, 4.7355712e-05],\n",
       "         [4.7350197e-05, 4.7306203e-05, 4.7349971e-05, ...,\n",
       "          4.7298865e-05, 4.7338232e-05, 4.7353577e-05],\n",
       "         ...,\n",
       "         [4.7350197e-05, 4.7306203e-05, 4.7349971e-05, ...,\n",
       "          4.7298865e-05, 4.7338232e-05, 4.7353577e-05],\n",
       "         [4.7350197e-05, 4.7306203e-05, 4.7349971e-05, ...,\n",
       "          4.7298865e-05, 4.7338232e-05, 4.7353577e-05],\n",
       "         [4.7350197e-05, 4.7306203e-05, 4.7349971e-05, ...,\n",
       "          4.7298865e-05, 4.7338232e-05, 4.7353577e-05]],\n",
       " \n",
       "        [[4.7318474e-05, 4.7281090e-05, 4.7302085e-05, ...,\n",
       "          4.7294132e-05, 4.7336627e-05, 4.7393027e-05],\n",
       "         [4.7333699e-05, 4.7279831e-05, 4.7323003e-05, ...,\n",
       "          4.7324545e-05, 4.7336609e-05, 4.7381982e-05],\n",
       "         [4.7348229e-05, 4.7308153e-05, 4.7350852e-05, ...,\n",
       "          4.7300229e-05, 4.7339458e-05, 4.7352791e-05],\n",
       "         ...,\n",
       "         [4.7348229e-05, 4.7308153e-05, 4.7350852e-05, ...,\n",
       "          4.7300229e-05, 4.7339458e-05, 4.7352791e-05],\n",
       "         [4.7348229e-05, 4.7308153e-05, 4.7350852e-05, ...,\n",
       "          4.7300229e-05, 4.7339458e-05, 4.7352791e-05],\n",
       "         [4.7348229e-05, 4.7308153e-05, 4.7350852e-05, ...,\n",
       "          4.7300229e-05, 4.7339458e-05, 4.7352791e-05]],\n",
       " \n",
       "        ...,\n",
       " \n",
       "        [[4.7314814e-05, 4.7285233e-05, 4.7303562e-05, ...,\n",
       "          4.7333200e-05, 4.7355818e-05, 4.7339774e-05],\n",
       "         [4.7324669e-05, 4.7260903e-05, 4.7335598e-05, ...,\n",
       "          4.7283211e-05, 4.7382615e-05, 4.7394067e-05],\n",
       "         [4.7319059e-05, 4.7285550e-05, 4.7327394e-05, ...,\n",
       "          4.7316720e-05, 4.7396577e-05, 4.7395522e-05],\n",
       "         ...,\n",
       "         [4.7346850e-05, 4.7313864e-05, 4.7351365e-05, ...,\n",
       "          4.7295114e-05, 4.7344587e-05, 4.7363734e-05],\n",
       "         [4.7346850e-05, 4.7313864e-05, 4.7351365e-05, ...,\n",
       "          4.7295114e-05, 4.7344587e-05, 4.7363734e-05],\n",
       "         [4.7346850e-05, 4.7313864e-05, 4.7351365e-05, ...,\n",
       "          4.7295114e-05, 4.7344587e-05, 4.7363734e-05]],\n",
       " \n",
       "        [[4.7337653e-05, 4.7275160e-05, 4.7321504e-05, ...,\n",
       "          4.7346930e-05, 4.7343055e-05, 4.7379188e-05],\n",
       "         [4.7334659e-05, 4.7275778e-05, 4.7333189e-05, ...,\n",
       "          4.7358837e-05, 4.7335823e-05, 4.7371923e-05],\n",
       "         [4.7361955e-05, 4.7322672e-05, 4.7340145e-05, ...,\n",
       "          4.7275302e-05, 4.7354759e-05, 4.7362373e-05],\n",
       "         ...,\n",
       "         [4.7361955e-05, 4.7322672e-05, 4.7340145e-05, ...,\n",
       "          4.7275302e-05, 4.7354759e-05, 4.7362373e-05],\n",
       "         [4.7361955e-05, 4.7322672e-05, 4.7340145e-05, ...,\n",
       "          4.7275302e-05, 4.7354759e-05, 4.7362373e-05],\n",
       "         [4.7361955e-05, 4.7322672e-05, 4.7340145e-05, ...,\n",
       "          4.7275302e-05, 4.7354759e-05, 4.7362373e-05]],\n",
       " \n",
       "        [[4.7328173e-05, 4.7285692e-05, 4.7320234e-05, ...,\n",
       "          4.7334062e-05, 4.7351819e-05, 4.7395170e-05],\n",
       "         [4.7345580e-05, 4.7291061e-05, 4.7327179e-05, ...,\n",
       "          4.7346221e-05, 4.7342117e-05, 4.7380105e-05],\n",
       "         [4.7323625e-05, 4.7282429e-05, 4.7335478e-05, ...,\n",
       "          4.7360292e-05, 4.7352551e-05, 4.7367481e-05],\n",
       "         ...,\n",
       "         [4.7361395e-05, 4.7324938e-05, 4.7340574e-05, ...,\n",
       "          4.7276248e-05, 4.7357607e-05, 4.7361649e-05],\n",
       "         [4.7361395e-05, 4.7324938e-05, 4.7340574e-05, ...,\n",
       "          4.7276248e-05, 4.7357607e-05, 4.7361649e-05],\n",
       "         [4.7361395e-05, 4.7324938e-05, 4.7340574e-05, ...,\n",
       "          4.7276248e-05, 4.7357607e-05, 4.7361649e-05]]], dtype=float32)>,\n",
       " 'weibo_premask_mlm': <tf.Tensor: shape=(8, 20, 21128), dtype=float32, numpy=\n",
       " array([[[4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         ...,\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05]],\n",
       " \n",
       "        [[4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         ...,\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05]],\n",
       " \n",
       "        [[4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         ...,\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05]],\n",
       " \n",
       "        ...,\n",
       " \n",
       "        [[4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         ...,\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05]],\n",
       " \n",
       "        [[4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         ...,\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05]],\n",
       " \n",
       "        [[4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         ...,\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05],\n",
       "         [4.737305e-05, 4.728283e-05, 4.738828e-05, ..., 4.725268e-05,\n",
       "          4.732817e-05, 4.732038e-05]]], dtype=float32)>}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hide\n",
    "tb.test_mtl_model(mtl_model=mmoe, include_top=False, mode=TRAIN)\n",
    "tb.test_mtl_model(mtl_model=mmoe, include_top=False, mode=EVAL)\n",
    "tb.test_mtl_model(mtl_model=mmoe, include_top=False, mode=PREDICT)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.3 64-bit ('base': conda)",
   "name": "python383jvsc74a57bd0b85cb1a4046041daefd6642872eb6047e6f7a68fd17c92cf8b50ede4b71c10f2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
